import urllib.request
import requests
import re,os
import random

def head():
    agentall = [
        'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Mobile Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/58.0',
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
        ]
    agent = random.choice(agentall)
    headers = ('User-Agent',agent)
    opener = urllib.request.build_opener()
    opener.addheaders = [headers]
    urllib.request.install_opener(opener)

#------------评论查询------------
def comment(commentid):

    namelist = []
    commentlist = []
    commentall = []

    pat = '"nickname":"(.*?)".*?"content":"(.*?)"'
    
    for i in range(0,2):
        #head()
        url = 'https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON&productId='+str(commentid)+'&score=0&sortType=5&page='+str(i)+'&pageSize=10'       
        #html = urllib.request.urlopen(url).read().decode('utf-8')
        html = requests.get(url).text
        comments= re.compile(pat,re.S).findall(html)
        for j in range(0,len(comments)):
            commentlist.append(comments[j])
    for i in range(0,len(commentlist)):
        commentall.append(commentlist[i][0]+':'+commentlist[i][1]+'\n')
    return commentall
        

#------------商品查询-------------
def summary(html):
    summaryall = []
    idpat = 'data-pid="(.*?)"'
    pricepat = '<em>￥</em><i>(.*?)</i>'
    namepat = '<title>【.*?】(.*?)【.*?】-京东</title>'
    namepat1 = '<title>(.*?)【.*?】-京东</title>'
    shoppat = 'dianpuname1">(.*?)</a>'
    commentpat = '">(.*?)</a>条评价'

    idlist = re.compile(idpat).findall(html)
    pricelist = re.compile(pricepat).findall(html)
    commentlist = re.compile(commentpat).findall(html)
    namelist = []
    shoplist = []
    for i in range(0,len(idlist)):
        head()
        url = 'https://item.jd.com/'+str(idlist[i])+'.html'
        data = requests.get(url).text
        try:
            name = re.compile(namepat).findall(data)
            namelist.append(name[0])
        except IndexError:
            name = re.compile(namepat1).findall(data)
            namelist.append(name[0])
        
        shop = re.compile(shoppat).findall(data)
        if len(shop) == 0:
            shoplist.append('京东自营')
        else:
            shoplist.append(shop[0])
            
    for j in range(0,len(namelist)):
        summaryall.append('{'+'"商品名":'+namelist[j]+','+'"商品价格":'+pricelist[j]+','+'"商品出售方":'+shoplist[j]+','+'"商品评论数":'+commentlist[j]+'}\n')
    
    return(summaryall,namelist,idlist)

#----------商品信息存储--------------
def savefile(summary):
    with open('商品信息.txt','a',encoding='utf-8') as f:
        f.writelines(summary)

#---------评论存储------------
def savecomment(comment,name):
    with open(name+'评论.txt','a',encoding='utf-8') as f:
        f.writelines(comment)
        
#------------评论标题检查-------------------
def title(thistitle):
    tag = ['\\', '/', ':', '*', '?', '《》', '<', '>', '|']
    try:
        for each in thistitle:
                if each in tag:
                    thistitle = thistitle.replace(each, '')
    except IndexError:
         pass
    return thistitle

#-------爬虫主程序----------
def main(path,page=40):
    os.chdir(path)
    os.mkdir('评论')
    a = 1       
    for i in range(0,page):
        os.chdir(path)
        try:
            head()
            url = 'https://search.jd.com/Search?keyword=%E9%9B%B6%E9%A3%9F&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&page='+str(a)
            html = urllib.request.urlopen(url).read().decode('utf-8')
            alllist=summary(html)
            savefile(alllist[0])
            os.chdir('评论')
            for j in range(0,len(alllist[2])):
                try:
                    commentlist = comment(alllist[2][j])
                    savecomment(commentlist,alllist[1][j])
                except OSError:
                    commentlist = comment(alllist[2][j])
                    thistitle = title(alllist[1][j])
                    savecomment(commentlist,thistitle)
            a+=2
            print('第%d页'%(i+1))
        except UnicodeEncodeError:
            pass
path = input("请输入信息存储文件夹(请将'\\'替换成'/')：")
main(path)
